package com.jngreen.crawler;

import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.slf4j.Logger;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import com.jngreen.util.CrawlerProxy;
import com.jngreen.util.ImageUtil;

@SuppressWarnings("rawtypes")
public class CrawlerTest implements PageProcessor{
	static Logger logger = org.slf4j.LoggerFactory.getLogger("com.jngreen.webmagic");
	private Site site = Site.me();
	private Map model;
	private List<String> indexUrls = null;
	SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

	public CrawlerTest(Map model){
		this.model = model;
		init();
	}
	
	public Map getModel() {
		return model;
	}

	public void setModel(Map model) {
		this.model = model;
	}

	public List<String> getIndexUrls() {
		return indexUrls;
	}

	public void setIndexUrls(List<String> indexUrls) {
		this.indexUrls = indexUrls;
	}

	private void init() {
		Map h = (Map)model.get(CrawlerConstants.TEMP_FIELD_HEADERS);
		for (Object e : h.entrySet()){
			Entry en = (Entry)e;
			getSite().addHeader((String)en.getKey(), (String)en.getValue());
		}
		getSite().setUserAgent((String)model.get(CrawlerConstants.TEMP_FIELD_USERAGENT));
		getSite().setRetryTimes((int)model.get(CrawlerConstants.TEMP_FIELD_RETRY))
				.setSleepTime((int)model.get(CrawlerConstants.TEMP_FIELD_SLEEP));
		String url1 = (String)model.get(CrawlerConstants.TEMP_FIELD_FIRSTINDEXPAGEURL);
		indexUrls = new ArrayList<>();
		indexUrls.add(url1);
		String otherUrl = (String)model.get(CrawlerConstants.TEMP_FIELD_OTHERINDEXPAGEURL);
		if (null != otherUrl){
			int pageNumIndex = otherUrl.indexOf(CrawlerConstants.INDEX_HOLDER);
			if (pageNumIndex != -1){
				String urlStart = otherUrl.substring(0,pageNumIndex);
				String urlEnd = otherUrl.substring(pageNumIndex + CrawlerConstants.INDEX_HOLDER.length());
				try{
					int start = (int)model.get(CrawlerConstants.TEMP_FIELD_PAGEINDEXSTART);
					int end = (int)model.get(CrawlerConstants.TEMP_FIELD_PAGEINDEXEND);
					for (int i = start; i <= end; i++){
						indexUrls.add(urlStart + i + urlEnd);
					}
				} catch(Throwable e){
					logger.error(CrawlerConstants.TEMP_FIELD_PAGEINDEXSTART + "或者" +
							CrawlerConstants.TEMP_FIELD_PAGEINDEXEND + "未设置");
					throw e;
				}
			}
		}
	}
	public static void main(String[] args) {
		
		Map model = YamlReader.readYaml("test.yaml");
		System.out.println("模板配置名称：" + model.get(CrawlerConstants.TEMP_FIELD_NAME));
		System.out.println("模板配置描述" + model.get(CrawlerConstants.TEMP_FIELD_DESC));
		System.out.println("索引页" + model.get(CrawlerConstants.TEMP_FIELD_FIRSTINDEXPAGEURL));
		System.out.println("其他索引页" + model.get(CrawlerConstants.TEMP_FIELD_OTHERINDEXPAGEURL));
		System.out.println("索引页起始页" + model.get(CrawlerConstants.TEMP_FIELD_PAGEINDEXSTART));
		System.out.println("索引页终止页" + model.get(CrawlerConstants.TEMP_FIELD_PAGEINDEXEND));
		System.out.println("提供方式" + model.get(CrawlerConstants.TEMP_FIELD_EXTRACTMODE));
		List l = (List)model.get(CrawlerConstants.TEMP_FIELD_CONTENTLOCATIONS);
		for (int i = 0; i < l.size(); i++){
			Map o = (Map)l.get(i);
			System.out.println("区域路径：" + o.get(CrawlerConstants.TEMP_FIELD_LOCATIONPATH));
			System.out.println("URL正则表达式：" + o.get(CrawlerConstants.TEMP_FIELD_URLPATTERN));
		}
		CrawlerTest t = new CrawlerTest(model);
		
		
		Spider spider = Spider.create(t)
				.startUrls(t.getIndexUrls())
//				.addUrl("http://roll.news.qq.com/interface/cpcroll.php?callback=rollback&site=sports&mode=1&cata=&date=2018-06-07&page=1&_=1528332163575")
				.thread(1);

		spider.run();
	}
	
	

	@Override
	public void process(Page page) {
		if (indexUrls.contains(page.getUrl().get())){
			if ("xpath".equals(model.get(CrawlerConstants.TEMP_FIELD_EXTRACTMODE))){
				List l = (List)model.get(CrawlerConstants.TEMP_FIELD_CONTENTLOCATIONS);
				for (int i = 0; i < l.size(); i++){
					Map o = (Map)l.get(i);
					Selectable s = page.getHtml().xpath((String)o.get(CrawlerConstants.TEMP_FIELD_LOCATIONPATH));
					List<String> list = s.links().regex((String)o.get(CrawlerConstants.TEMP_FIELD_URLPATTERN)).all();
					System.out.println(list);
					page.addTargetRequests(list);
				}
			} else {
				//css选择器的方式稍后实现
			}
		} else {
			List fields = (List)this.model.get(CrawlerConstants.TEMP_FIELD_FIELDS);
			for (int i = 0; i < fields.size(); i++){
				Map m = (Map)fields.get(i);
				if (!m.containsKey(CrawlerConstants.TEMP_FIELD_NAME)){
					logger.error(CrawlerConstants.TEMP_FIELD_NAME + "字段没有设置");
					page.setSkip(true);
					return;
				}
				if (!m.containsKey(CrawlerConstants.TEMP_FIELD_TYPE)){
					logger.error(CrawlerConstants.TEMP_FIELD_TYPE + "字段没有设置");
					page.setSkip(true);
					return;
				}
				String name = m.get(CrawlerConstants.TEMP_FIELD_NAME).toString();
				String type = m.get(CrawlerConstants.TEMP_FIELD_TYPE).toString();
				if (CrawlerConstants.TEMP_FIELD_SEARCH.equals(type)){
					if (!m.containsKey(CrawlerConstants.TEMP_FIELD_PREFIX)){
						logger.error(CrawlerConstants.TEMP_FIELD_SEARCH +
								"方式提取字段时，没有设置" + CrawlerConstants.TEMP_FIELD_PREFIX);
						page.setSkip(true);
						return;
					}
					if (!m.containsKey(CrawlerConstants.TEMP_FIELD_POST)){
						logger.error(CrawlerConstants.TEMP_FIELD_SEARCH +
								"方式提取字段时，没有设置" + CrawlerConstants.TEMP_FIELD_POST);
						page.setSkip(true);
						return;
					}
					String c = page.getRawText();
					String prefix = m.get(CrawlerConstants.TEMP_FIELD_PREFIX).toString();
					int indexStart = c.indexOf(prefix);
					if (indexStart == -1){
						logger.error(CrawlerConstants.TEMP_FIELD_SEARCH +
								"方式提取字段时，找不到前缀" + prefix);
						page.setSkip(true);
						return;
					}
					int indexEnd = c.indexOf(m.get(CrawlerConstants.TEMP_FIELD_POST).toString(),
							indexStart + prefix.length());
					if (indexEnd == -1){
						logger.error(CrawlerConstants.TEMP_FIELD_SEARCH +
								"方式提取字段时，找不到后缀" + m.get(CrawlerConstants.TEMP_FIELD_POST));
						page.setSkip(true);
						return;
					}
					
					page.putField(name, c.substring(indexStart + prefix.length(), indexEnd).trim());
				} else {
					if (!m.containsKey(CrawlerConstants.TEMP_FIELD_VALUE)){
						logger.error(type + "方式提取字段时，没有设置value");
						page.setSkip(true);
						return;
					}
					if (CrawlerConstants.TEMP_FIELD_XPATH.equals(type)) {
						String value = m.get(CrawlerConstants.TEMP_FIELD_VALUE).toString();
						String s = page.getHtml().xpath(value).get();
						if (s == null || "".equals(s)){
							logger.info("爬虫" + model.get(CrawlerConstants.TEMP_FIELD_NAME) +"未找到字段 " + 
									name + "的值");
						}
						page.putField(name, s);
					} else if (CrawlerConstants.TEMP_FIELD_CSS.equals(type)) {
						
					} else if (CrawlerConstants.TEMP_FIELD_STATIC.equals(type)) {
						Object value = m.get(CrawlerConstants.TEMP_FIELD_VALUE);
						page.putField(name, value);
					} else if (CrawlerConstants.TEMP_FIELD_SYS.equals(type)) {
						String value = m.get(CrawlerConstants.TEMP_FIELD_VALUE).toString();
						Object res = getSysField(value, page);
						page.putField(name, res);
					} else if (CrawlerConstants.TEMP_FIELD_REGEX.equals(type)){
						String value = m.get(CrawlerConstants.TEMP_FIELD_VALUE).toString();
						Pattern p = null;
						try{
							p = Pattern.compile(value);
						} catch (PatternSyntaxException e){
							logger.error(type + "方式提取字段时，正则表达式语法错误: " + value);
						page.setSkip(true);
						}
						String c = page.getRawText();
						Matcher matcher = p.matcher(c);
						if (matcher.find() && matcher.groupCount()>0){
							page.putField(name, matcher.group(1));
						} else {
							logger.info(type + "方式提取字段时，正则表达式未找到结果，或者未添加group: " + 
									value);
						}
					}
				}
				
				if (m.containsKey(CrawlerConstants.TEMP_FIELD_REMOVELINESEG)
						&& CrawlerConstants.FIELD_VALUE_Y.equals(m.get(CrawlerConstants.TEMP_FIELD_REMOVELINESEG))){
					String s = page.getResultItems().get(name);
					s = s.replaceAll("\\r|\\n", "");
					page.putField(name, s);
				}
				
				if (m.containsKey(CrawlerConstants.TEMP_FIELD_ISPICTURE)
						&& CrawlerConstants.FIELD_VALUE_Y.equals(m.get(CrawlerConstants.TEMP_FIELD_ISPICTURE))){
					String tp = page.getResultItems().get(name);
					if (tp != null){
						byte[] imageBytes = null;
						try {
							imageBytes = ImageUtil.downLoadImageByUrl(
									ImageUtil.toRealUrl(new URL(page.getUrl().get()), tp));
						} catch (Exception e) {
						}
						page.putField(name,imageBytes);
					}
				}
			}
		}
	}

	private Object getSysField(String value, Page page) {
		if ("pageUrl".equals(value)){
			return page.getUrl().get();
		} else if ("timestamp".equals(value)){
			return format.format(new Date());
		} else if ("rawText".equals(value)){
			return page.getRawText();
		}
		return null;
	}

	@Override
	public Site getSite() {
		CrawlerProxy.setProxy(site);
		return site;
	}

}
